package com.jvm123.hkmte

import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements



def getPageContent(String url, FileWriter writer) {

    int tryTimes = 0

    Document doc = null
    while (doc == null) {
        tryTimes += 1
        try {
            doc = Jsoup.connect(url).get()
        } catch (Exception ignored) { }
        if (doc == null) {
            Thread.sleep(2000)
            println "======== has try ${tryTimes} times =========="
        }
        if (tryTimes > 6) {
            println "======== has try ${tryTimes} times and no more try =========="
            return
        }
    }
    // div
    Elements eleList = doc.getElementsByClass("col-md-6 col-sm-4 col-xs-3")
    for (Element li : eleList.findAll()) {
        try {
            Elements div = li.getElementsByClass("myui-vodlist__detail")
            // a
            String a = div.get(0).getElementsByTag("a").get(0)
            String p = div.get(0).getElementsByTag("p").get(0)
            println a
            writer.write("\n")
            writer.write(a.replace("\n", " "))
            writer.write(p.replace("\n", " "))
        } catch (Exception ex) {
            ex.printStackTrace()
        }
    }
}

File newFile(String fileName) {
    File file = new File(fileName)
    if (!file.exists()) {
        file.createNewFile()
    }
    file
}


File file = newFile("hkmte.com.txt")
FileWriter fw = new FileWriter(file)
for (i in 1 .. 843) { //
    String url = "http://www.hkmte.com/list/?1-${i}.html"
    getPageContent(url, fw)
    println i
}
fw.close()
